/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

/*
 * UI.java
 *
 * Created on Oct 27, 2010, 9:36:20 PM
 */
package indexer;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.String;
import java.net.Socket;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.print.DocFlavor.READER;
import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTMLEditorKit.ParserCallback;
import javax.swing.text.html.*;
import javax.swing.text.html.HTML.Tag;
import javax.swing.text.html.parser.ParserDelegator;
import java.io.IOException;
import java.io.FileReader;
import java.io.Reader;
import java.util.List;
import java.util.ArrayList;

import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import javax.swing.text.html.parser.ParserDelegator;
import javax.swing.text.html.HTMLEditorKit.ParserCallback;
import javax.swing.text.html.HTML.Tag;
import javax.swing.text.html.HTML.Attribute;
import javax.swing.text.MutableAttributeSet;
import dnstry.*;
import org.omg.PortableInterceptor.SYSTEM_EXCEPTION;

/**
 *
 * @author Claudiu
 */
class UI extends javax.swing.JFrame {

    Socket sock;
    FileWriter fstream;
    BufferedWriter file;
    BufferedReader input;
    DataOutputStream output;
    BufferedWriter file_;
    String redirect;
    LinkedList<String> links;
    String textFileName;
    ArrayList<String> list;
    int contor;
    String currentFile;
    DnsQuery dnsQuery;
    int maxSize = 25;
    int timeout;

    /** Creates new form UI */
    public UI() {
        initComponents();
        list = new ArrayList<String>();
        dnsQuery = new DnsQuery();
        timeout = 5000;
        contor = 0;

    }

    /** This method is called from within the constructor to
     * initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is
     * always regenerated by the Form Editor.
     */
    @SuppressWarnings("unchecked")
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        jScrollPane1 = new javax.swing.JScrollPane();
        Display = new javax.swing.JTextArea();
        uriField = new javax.swing.JTextField();
        GoButton = new javax.swing.JButton();
        jScrollPane2 = new javax.swing.JScrollPane();
        DisplayLinks = new javax.swing.JTextArea();
        jScrollPane3 = new javax.swing.JScrollPane();
        DisplayText = new javax.swing.JTextArea();
        jScrollPane4 = new javax.swing.JScrollPane();
        DisplayMeta = new javax.swing.JTextArea();
        DisplayTitle = new javax.swing.JTextField();
        proxyField = new javax.swing.JTextField();
        proxyBox = new javax.swing.JCheckBox();
        getOverrideBox = new javax.swing.JCheckBox();
        maxDepthField1 = new javax.swing.JTextField();
        jLabel1 = new javax.swing.JLabel();
        SearchBox = new javax.swing.JTextField();
        SearchBtn = new javax.swing.JButton();
        DnsAddressBox = new javax.swing.JTextField();
        jLabel2 = new javax.swing.JLabel();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);

        Display.setBackground(new java.awt.Color(0, 0, 0));
        Display.setColumns(20);
        Display.setForeground(new java.awt.Color(204, 204, 204));
        Display.setRows(5);
        Display.setText("http response ");
        jScrollPane1.setViewportView(Display);

        uriField.setBackground(new java.awt.Color(0, 0, 0));
        uriField.setForeground(new java.awt.Color(240, 240, 240));
        uriField.setText("www.ace.tuiasi.ro");
        uriField.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                uriFieldActionPerformed(evt);
            }
        });

        GoButton.setText("Go");
        GoButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                GoButtonActionPerformed(evt);
            }
        });

        DisplayLinks.setBackground(new java.awt.Color(0, 0, 0));
        DisplayLinks.setColumns(20);
        DisplayLinks.setForeground(new java.awt.Color(204, 204, 204));
        DisplayLinks.setRows(5);
        DisplayLinks.setText("links ");
        jScrollPane2.setViewportView(DisplayLinks);

        DisplayText.setBackground(new java.awt.Color(0, 0, 0));
        DisplayText.setColumns(20);
        DisplayText.setForeground(new java.awt.Color(204, 204, 204));
        DisplayText.setRows(5);
        DisplayText.setText("text");
        jScrollPane3.setViewportView(DisplayText);

        DisplayMeta.setBackground(new java.awt.Color(0, 0, 0));
        DisplayMeta.setColumns(20);
        DisplayMeta.setForeground(new java.awt.Color(204, 204, 204));
        DisplayMeta.setRows(5);
        DisplayMeta.setText("meta ");
        jScrollPane4.setViewportView(DisplayMeta);

        DisplayTitle.setBackground(new java.awt.Color(0, 0, 0));
        DisplayTitle.setForeground(new java.awt.Color(204, 204, 204));
        DisplayTitle.setText("title");
        DisplayTitle.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                DisplayTitleActionPerformed(evt);
            }
        });

        proxyField.setBackground(new java.awt.Color(0, 0, 0));
        proxyField.setForeground(new java.awt.Color(240, 240, 240));
        proxyField.setText("proxy.tuiasi.ro");

        proxyBox.setText("goProxy");

        getOverrideBox.setText("attempt get even if not allowed");

        maxDepthField1.setBackground(new java.awt.Color(0, 0, 0));
        maxDepthField1.setForeground(new java.awt.Color(240, 240, 240));
        maxDepthField1.setText("25");

        jLabel1.setText("Link depth...");

        SearchBox.setBackground(new java.awt.Color(0, 0, 0));
        SearchBox.setForeground(new java.awt.Color(240, 240, 240));

        SearchBtn.setText("Search");
        SearchBtn.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                SearchBtnActionPerformed(evt);
            }
        });

        DnsAddressBox.setBackground(new java.awt.Color(0, 0, 0));
        DnsAddressBox.setForeground(new java.awt.Color(240, 240, 240));
        DnsAddressBox.setText("8.8.8.8");

        jLabel2.setText("Dns Address :");

        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(jScrollPane4, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 734, Short.MAX_VALUE)
                            .addComponent(jScrollPane1, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 734, Short.MAX_VALUE)
                            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
                                .addComponent(SearchBox, javax.swing.GroupLayout.DEFAULT_SIZE, 663, Short.MAX_VALUE)
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                .addComponent(SearchBtn))
                            .addGroup(layout.createSequentialGroup()
                                .addComponent(jLabel2)
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
                                .addComponent(DnsAddressBox, javax.swing.GroupLayout.PREFERRED_SIZE, 195, javax.swing.GroupLayout.PREFERRED_SIZE)))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
                            .addComponent(jScrollPane3, javax.swing.GroupLayout.DEFAULT_SIZE, 382, Short.MAX_VALUE)
                            .addComponent(DisplayTitle, javax.swing.GroupLayout.DEFAULT_SIZE, 382, Short.MAX_VALUE)
                            .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 382, javax.swing.GroupLayout.PREFERRED_SIZE)))
                    .addGroup(layout.createSequentialGroup()
                        .addComponent(uriField, javax.swing.GroupLayout.PREFERRED_SIZE, 264, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(proxyField, javax.swing.GroupLayout.PREFERRED_SIZE, 264, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addGap(18, 18, 18)
                        .addComponent(jLabel1, javax.swing.GroupLayout.DEFAULT_SIZE, 118, Short.MAX_VALUE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(maxDepthField1, javax.swing.GroupLayout.PREFERRED_SIZE, 63, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addGap(18, 18, 18)
                        .addComponent(proxyBox)
                        .addGap(10, 10, 10)
                        .addComponent(getOverrideBox)
                        .addGap(53, 53, 53)
                        .addComponent(GoButton, javax.swing.GroupLayout.PREFERRED_SIZE, 62, javax.swing.GroupLayout.PREFERRED_SIZE)))
                .addContainerGap())
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(layout.createSequentialGroup()
                        .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 185, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(jScrollPane3, javax.swing.GroupLayout.DEFAULT_SIZE, 349, Short.MAX_VALUE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(DisplayTitle, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                    .addGroup(layout.createSequentialGroup()
                        .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 409, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(jScrollPane4, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(SearchBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(SearchBtn))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(jLabel2)
                            .addComponent(DnsAddressBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(uriField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(GoButton)
                    .addComponent(proxyField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(proxyBox)
                    .addComponent(getOverrideBox)
                    .addComponent(maxDepthField1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jLabel1, javax.swing.GroupLayout.PREFERRED_SIZE, 22, javax.swing.GroupLayout.PREFERRED_SIZE))
                .addGap(26, 26, 26))
        );

        pack();
    }// </editor-fold>//GEN-END:initComponents

    public static List<String> extractLinks(Reader reader) throws IOException {
        final ArrayList<String> list = new ArrayList<String>();

        ParserDelegator parserDelegator = new ParserDelegator();
        ParserCallback parserCallback = new ParserCallback() {

            public void handleText(final char[] data, final int pos) {
            }

            public void handleStartTag(Tag tag, MutableAttributeSet attribute, int pos) {
                if (tag == Tag.A) {
                    String address = (String) attribute.getAttribute(Attribute.HREF);
                    list.add(address);
                }
            }

            public void handleEndTag(Tag t, final int pos) {
            }

            public void handleSimpleTag(Tag t, MutableAttributeSet a, final int pos) {
            }

            public void handleComment(final char[] data, final int pos) {
            }

            public void handleError(final java.lang.String errMsg, final int pos) {
            }
        };
        parserDelegator.parse(reader, parserCallback, false);
        return list;
    }

    public /* List<String>*/ void extractText(Reader reader) throws IOException {
        final ArrayList<String> list_text = new ArrayList<String>();
        final ArrayList<String> list_links = new ArrayList<String>();
        final ArrayList<String> list_meta = new ArrayList<String>();
        final ArrayList<String> list_title = new ArrayList<String>();
        final ArrayList<String> list_aux = new ArrayList<String>();
        final ArrayList<String> myTag = new ArrayList<String>();
        ParserDelegator parserDelegator = new ParserDelegator();
        ParserCallback parserCallback = new ParserCallback() {

            public void handleText(final char[] data, final int pos) {
                String aux = new String(data);
                if ((!aux.contains("<")) && (!aux.contains(">"))) {
                    list_text.add(aux);
                }
                if (myTag.size() > 0) {
                    if (myTag.get(0).contains("title")) {
                        list_title.add("title :" + aux);
                    }
                }
            }

            public void handleStartTag(Tag tag, MutableAttributeSet attribute, int pos) {
                myTag.add("title   " + tag.toString());

                if ((tag == Tag.A) || (tag == Tag.LINK)) {
                    String address = (String) attribute.getAttribute(Attribute.HREF);
                    if (address != null) {
                        if ((address.contains("#")) && (address.length() > 1)) {
                            String aux = address;
                            address = aux.substring(0, aux.indexOf('#'));
                        }
                        if (address.toLowerCase().startsWith("http")) {
                            list_links.add(address);
                        } else if ((address.contains("http")) && (address.startsWith("http"))) {
                            list_links.add(address.substring(address.indexOf("h")));
                        } else {
                            if (address.startsWith("./") || address.startsWith("../")) {
                                list_links.add("http://" + uriField.getText() + address.substring(address.indexOf("/")));
                            } else {
                                list_links.add("http://" + uriField.getText() + "/" + address);
                            }
                        }
                    }
                } else {

                    if (tag == tag.META) {
                        list_meta.add("found meta");
                    }

                }

            }

            public void handleEndTag(Tag t, final int pos) {
                myTag.set(0, "0");
            }

            public void handleSimpleTag(Tag tag, MutableAttributeSet attribute, final int pos) {
                if ((tag == Tag.A) || (tag == Tag.LINK)) {
                    String address = (String) attribute.getAttribute(Attribute.HREF);
                    if (address != null) {
                        if ((address.contains("#")) && (address.length() > 1)) {
                            String aux = address;
                            address = aux.substring(0, aux.indexOf('#'));
                        }
                        if (address.toLowerCase().startsWith("http")) {
                            list_links.add(address);
                        } else if ((address.contains("http")) && (address.startsWith("http"))) {
                            list_links.add(address.substring(address.indexOf("h")));
                        } else {
                            if (address.startsWith("./") || address.startsWith("../")) {
                                list_links.add("http://" + uriField.getText() + address.substring(address.indexOf("/")));
                            } else {
                                list_links.add("http://" + uriField.getText() + "/" + address);
                            }
                        }
                    }
                } else {

                    if (tag == tag.META) {
                        if (attribute.getAttribute(Attribute.NAME) != null) {
                            if (((String) attribute.getAttribute(Attribute.NAME)).contains("keywords")) {
                                list_meta.add("keywords :" + (String) attribute.getAttribute(Attribute.CONTENT));
                            } else if (((String) attribute.getAttribute(Attribute.NAME)).contains("description")) {
                                list_meta.add("description :" + (String) attribute.getAttribute(Attribute.CONTENT));
                            } else if (((String) attribute.getAttribute(Attribute.NAME)).contains("robots")) {
                                list_meta.add("robots :" + (String) attribute.getAttribute(Attribute.CONTENT));
                            }

                        }
                    } else if (tag == tag.TITLE) {

                        list_title.add((String) attribute.getAttribute(Attribute.TEXT));
                    }
                }

            }

            public void handleTitle(final char[] data) {
            }

            public void handleError(final java.lang.String errMsg, final int pos) {
            }
        };
        parserDelegator.parse(reader, parserCallback, true);
        //Create a HashSet which allows no duplicates
        HashSet hashSet = new HashSet(list_links);

        //Assign the HashSet to a new ArrayList
        ArrayList<String> arrayList2 = new ArrayList(hashSet);

        //Ensure correct order, since HashSet doesn't
        Collections.sort(arrayList2);
        FileWriter fstreamLink = new FileWriter("links.txt");
        BufferedWriter fileLink = new BufferedWriter(fstreamLink);
        FileWriter fstreamText = new FileWriter(textFileName);
        BufferedWriter fileText = new BufferedWriter(fstreamText);
        fileLink.append("Links : \n");
        for (String line : arrayList2) {
            //  System.out.println("link line " + line);
            DisplayLinks.append(line + "\n");
            fileLink.append(line + "\n");
            if (!links.contains(line)) {
                links.add(line);
                System.out.println(line);
            }
        }
        // fileText.append("Text: \n");
        for (String line2 : list_text) {
            //    System.out.println("text line " + line2);
            DisplayText.append(line2 + "\n");
            fileText.append(line2 + "\n");
        }
        fileLink.append("Meta : \n");
        for (String line3 : list_meta) {
            //    System.out.println("text line " + line2);
            DisplayMeta.append(line3 + "\n");
            fileLink.append(line3 + "\n");
        }
        fileLink.append("Title : \n");
        for (String line4 : list_title) {
            // System.out.println("title line " + line4);
            if (line4 != null) {
                DisplayTitle.setText(line4 + "\n");
            }
            fileLink.append(line4);
        }
        fileLink.close();
        fileText.close();

    //return list;
    }

    void parseItGetOpt(String address) {
        try {
            String myField = address;

            String resource;
            if (myField.contains("/")) {
                if (myField.contains("http://")) {
                    myField = myField.substring(myField.indexOf("/") + 2);

                    try {
                        address = myField.substring(0, myField.indexOf("/"));
                        resource = myField.substring(myField.indexOf("/"));
                    } catch (Exception e) {
                        address = myField.substring(0, myField.length());
                        resource = "/";
                    }

                } else {
                    address = myField.substring(0, myField.indexOf("/"));
                    resource = myField.substring(myField.indexOf("/"));
                }
            } else {
                address = myField;
                resource = "/";
            }

            // httpGet(address, resource);

            Display.append("\n\n\n\n\n\n**********************************options");
            boolean canGet = httpOptions(address, resource);
            //boolean canGet = true;
            Display.append("\n\n\n\n\n\n**********************************head  robots");
            int robotsResponseCode = httpHead(address, "/robots.txt");
            //Display.append("\n\n\n\n\n\n**********************************head get");
            int headResponseCode = httpHead(address, resource);


            if (((canGet) && (headResponseCode == 200))) {
                Display.append("\n\n\n\n\n\n**********************************get");
                httpGet(address, resource);
            } else {

                if (!canGet) {
                    Display.append("get not allowed here ....i guess....");
                }
                if ((headResponseCode < 400) && (headResponseCode > 299)) {
                    //treat redirect
                    Display.append("\n\n\n\n\n\n**********************************redirected to " + redirect);
                    redirect();
                }
            }
            if (robotsResponseCode == 200) {
                Display.append("\n\n\n\n\n\n**********************************get robots");
                httpGet(address, "/robots.txt");
            }
        } catch (Exception exc) {
            System.out.println(exc.getLocalizedMessage());
            exc.printStackTrace();
        }

    }

    void parseIt(String address) {
        try {
            String myField = address;

            String resource;
            if (myField.contains("/")) {
                if (myField.contains("http://")) {
                    myField = myField.substring(myField.indexOf("/") + 2);

                    try {
                        address = myField.substring(0, myField.indexOf("/"));
                        resource = myField.substring(myField.indexOf("/"));
                    } catch (Exception e) {
                        address = myField.substring(0, myField.length());
                        resource = "/";
                    }

                } else {
                    address = myField.substring(0, myField.indexOf("/"));
                    resource = myField.substring(myField.indexOf("/"));
                }
            } else {
                address = myField;
                resource = "/";
            }

            httpGet(address, resource);
        } catch (Exception exc) {
            System.out.println(exc.getLocalizedMessage());
            exc.printStackTrace();
        }

    }

    private String[] allowedMethodes(String input) {
        String[] retList = new String[10];
        String aux;
        String aux2;
        boolean stop = false;
        int initialPosition = input.indexOf(':') + 1;
        aux = input.substring(initialPosition);
        retList = aux.split(",");

        return retList;
    }

    private boolean httpOptions(String address, String resource) {
        String[] methods;
        boolean ret_val = false;
        if (resource == "/") {
            resource = "*";
        }
//        String myField = uriField.getText();
//        String address;
//        String resource;
//        if (myField.contains("/")) {
//            if (myField.contains("http://")) {
//                myField = myField.substring(myField.indexOf("/") + 2);
//                address = myField.substring(0, myField.indexOf("/"));
//                resource = myField.substring(myField.indexOf("/"));
//            } else {
//                address = myField.substring(0, myField.indexOf("/"));
//                resource = myField.substring(myField.indexOf("/"));
//            }
//        } else {
//            address = myField;
//            resource = "*";
//        }
        try {
            fstream = new FileWriter("fileopt.html");
            file = new BufferedWriter(fstream);
            Display.append("Initializing...\n");
            System.out.println("Initializing...");
            if (!proxyBox.isSelected()) {
                try {
                    sock = new Socket(dnsQuery.getIneAddress(address, DnsAddressBox.getText()), 80);
                    sock.setSoTimeout(timeout);
                } catch (Exception e) {
                    sock = new Socket(address, 80);
                    sock.setSoTimeout(timeout);
                }
            } else {
                sock = new Socket(proxyField.getText(), 8080);
                sock.setSoTimeout(timeout);
            }
            System.out.println("have socket...");
            Display.append("Have socket...\n");
            InputStreamReader in = new InputStreamReader(sock.getInputStream());
            input = new BufferedReader(in);
            output = new DataOutputStream(sock.getOutputStream());
            Display.append("Initialized...\n");
            System.out.println("Initialized...");
        } catch (Exception e) {
            System.out.println(e);
            Display.append("1" + e.getLocalizedMessage());
        }
        if (sock != null && output != null && input != null) {
            try {
                output.writeBytes("OPTIONS " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
                //  System.out.println("sent req");
                Display.append("OPTIONS " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");

                String response = input.readLine();
                while (response != null) {
                    Display.append(response + "\r\n");
                    //           System.out.println("have response - opt ");
                    if (input.ready()) {
                        response = input.readLine();
                        if (response.contains("Allow")) {
                            System.out.println("found allow");
                            methods = allowedMethodes(response);
                            for (int i = 0; i < methods.length; i++) {
                                System.out.println("method :" + methods[i]);
                                if (methods[i].contains("GET")) {
                                    ret_val = true;
                                }
                            }
                        }
                    //            System.out.println(response);
                    } else {
                        response = null;
                    }
                }
                //       System.out.println("out of loop");
                sock.close();
                file.close();
                Display.append("Nuked connection \n");
            } catch (Exception e) {
                System.out.println(e);
                Display.append("2" + e.getLocalizedMessage());
            }
        }

        return ret_val;

    }

    private void httpGet(String address, String resource) {

        try {

            fstream = new FileWriter("file" + address + +contor + ".html");
            file = new BufferedWriter(fstream);
            Display.append("Initializing...\n");
            System.out.println("Initializing...");
            // sock = new Socket("proxy.tuiasi.ro", 8080);
            try {
                sock = new Socket(dnsQuery.getIneAddress(address, DnsAddressBox.getText()), 80);
                sock.setSoTimeout(timeout);
            } catch (Exception e) {

                System.out.println("++++++++++++++++++++++++++++++++++++++++1111 DNS FUILURE ....attempting to use java built-in module and reinitialize ....");

                dnsQuery.nukeSocket();
                dnsQuery= new DnsQuery();
                contor--;
                e.printStackTrace();

            }
            // sock = new Socket(uriField.getText(), 80);
            System.out.println("have socket...");
            Display.append("Have socket...\n");
            InputStreamReader in = new InputStreamReader(sock.getInputStream());
            input = new BufferedReader(in);
            output = new DataOutputStream(sock.getOutputStream());
            Display.append("Initialized...\n");
            System.out.println("Initialized...");
        } catch (Exception e) {
            System.out.println(e.getMessage());
            e.printStackTrace();
            Display.append(e.getLocalizedMessage());
        }
        if (sock != null && output != null && input != null) {
            try {

                // sock = new Socket(uriField.getText(), 80);
                //sock = new Socket(address, 80);
                System.out.println("have socket...");
                Display.append("have socket \n");
                InputStreamReader in = new InputStreamReader(sock.getInputStream());
                input = new BufferedReader(in);
                output = new DataOutputStream(sock.getOutputStream());
                // output.writeBytes("GET / HTTP/1.1\r\nHost: " + uriField.getText() + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
                output.writeBytes("GET " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
                //Display.append("GET / HTTP/1.1\r\nHost: " + uriField.getText() + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
                Display.append("GET " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");

            } catch (Exception e) {
                System.out.println(e.getMessage());
                e.printStackTrace();
                Display.append(e.getLocalizedMessage());
            }
        }

        if (sock != null && output != null && input != null) {
            try {
                boolean fileFlag = false;
                String response = input.readLine();
                while (response != null) {
                    Display.append(response + "\r\n");
                    //     System.out.println("have response -get ");

                    if (input.ready()) {
                        response = input.readLine();

                        //System.out.println(response);
                        if (fileFlag == false) {
                            if ((response.contains("<html")) || (resource.contains("robots"))) {
                                fileFlag = true;
                            //     System.out.println("flag true");
                            }
                        } else {
                            file.write(response);
                        //      System.out.println("wrote line");
                        }
                    } else {
                        response = null;
                    }
                }
                //     System.out.println("out of loop");
                file.close();
                FileReader reader = new FileReader("file" + address + contor + ".html");
                //  List<String> lines = extractText(reader);
                // for (String line : lines) {
                //   System.out.println("text line " +line);
                // }
                //   lines= null;
                //textFileName= "text" + contor + " .txt";
                textFileName = "text" + address + contor + " .txt";
                //  List<String> lines = extractText(reader);
                // for (String line : lines) {
                //   System.out.println("text line " +line);
                // }
                //   lines= null;
                extractText(reader);
                list.add(textFileName);

//sock.close();
            } catch (Exception e) {

                System.out.println(e.getMessage());
                e.printStackTrace();

            }
        }
    }

    private int httpHead(String address, String resource) {
        int ret_val = 0;
        try {
            Display.append("Reinitializing connection \n");
            //sock = new Socket("proxy.cs.tuiasi.ro", 8080);
            // sock = new Socket(uriField.getText(), 80);
            if (!proxyBox.isSelected()) {
                try {
                    sock = new Socket(dnsQuery.getIneAddress(address, DnsAddressBox.getText()), 80);
                    sock.setSoTimeout(timeout);
                } catch (Exception e) {
                    sock = new Socket(address, 80);
                    sock.setSoTimeout(timeout);
                }
            } else {
                sock = new Socket(proxyField.getText(), 8080);
                sock.setSoTimeout(timeout);
            }
            System.out.println("have socket...");
            Display.append("have socket \n");
            InputStreamReader in = new InputStreamReader(sock.getInputStream());
            input = new BufferedReader(in);
            output = new DataOutputStream(sock.getOutputStream());
            // output.writeBytes("GET / HTTP/1.1\r\nHost: " + uriField.getText() + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
            output.writeBytes("HEAD " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
            //Display.append("GET / HTTP/1.1\r\nHost: " + uriField.getText() + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");
            Display.append("HEAD " + resource + " HTTP/1.1\r\nHost: " + address + "\r\nUser-Agent: CLIENT-Riw\r\n\r\n");

        } catch (Exception e) {
            System.out.println(e);
            Display.append("6" + e.getLocalizedMessage());
        }

        if (sock != null && output != null && input != null) {
            try {
                boolean fileFlag = false;
                String response = input.readLine();
                while (response != null) {
                    Display.append(response + "\r\n");
                    //     System.out.println("have response -get ");
                    if (response.contains("HTTP")) {
                        int v1, v2;

                        v1 = response.indexOf(" ");
                        v2 = response.indexOf(" ", v1 + 1);
                        String aux = response.substring(v1 + 1, v2);
                        try {
                            ret_val = Integer.parseInt(aux);
                        } catch (Exception e) {
                            System.out.println("-" + aux + "-not number" + v1 + "   " + v2);
                        }
                    }
                    if (input.ready()) {
                        response = input.readLine();
                        if ((response.contains("Location:"))) {
                            System.out.println("found location");
                            redirect = response.substring(response.indexOf("Location") + 9);
                        }
                        //System.out.println(response);
                        if (fileFlag == false) {
                            if (response.contains("<html")) {
                                fileFlag = true;
                            //     System.out.println("flag true");
                            }
                        } else {
                            file.write(response);
                        //      System.out.println("wrote line");
                        }
                    } else {
                        response = null;
                    }
                }
                //     System.out.println("out of loop");
                //     file.close();
         //       FileReader reader = new FileReader("file.html");
                //  List<String> lines = extractText(reader);
                // for (String line : lines) {
                //   System.out.println("text line " +line);
                // }
                //   lines= null;
             //   extractText(reader);


            } catch (Exception e) {
                System.out.println(e);
                Display.append("--" + e.getLocalizedMessage());
            }
        }
        return ret_val;
    }

    private void redirect() {
        String myField = redirect;
        String address;
        String resource;
        if (myField.contains("/")) {
            if (myField.contains("http://")) {
                myField = myField.substring(myField.indexOf("/") + 2);
                address = myField.substring(0, myField.indexOf("/"));
                resource = myField.substring(myField.indexOf("/"));
            } else {
                address = myField.substring(0, myField.indexOf("/"));
                resource = myField.substring(myField.indexOf("/"));
            }
        } else {
            address = myField;
            resource = "/";
        }
        Display.append("\n\n\n\n\n\n**********************************options");
        boolean canGet = httpOptions(address, resource);

        Display.append("\n\n\n\n\n\n**********************************head get");
        int headResponseCode = httpHead(address, resource);


        if ((canGet) || (getOverrideBox.isSelected())) {
            Display.append("\n\n\n\n\n\n**********************************get in redirect  ");
            httpGet(address, resource);
        } else {
            if (!canGet) {
                Display.append("get not allowed here ....i guess....");
            }
            if ((headResponseCode < 400) && (headResponseCode > 299)) {
                //treat redirect
                Display.append("redirected to " + redirect);
                //    httpGet(redirect, resource);
                redirect();
            }
        }






    }

    private String GetFileName(String fileName) {
        String file;
        String aux;
        file = fileName;
        if (file.contains("/")) {
            aux = file.replace("/", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.contains("\\")) {
            aux = file.replace("\\", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.contains("?")) {
            aux = file.replace("?", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.contains("<")) {
            aux = file.replace("<", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.contains(">")) {
            aux = file.replace(">", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.contains("|")) {
            aux = file.replace("|", " ");
        } else {
            aux = file;
        }
        file = aux;
        if (file.length() > 20) {
            file = file.substring(0, 20);
        }
        System.out.println("--" + file);
        return file;
    }
    private void GoButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_GoButtonActionPerformed
        // TODO add your handling code here:
        // if (uriField.getText() != "") {
        System.out.println("starting ....");
        boolean getOverride = getOverrideBox.isSelected();
        maxSize = Integer.parseInt(maxDepthField1.getText());
        if (uriField.getText() != "") {
            // parseIt(uriField.getText());
            // }String myField= uriField.getText();
            links = new LinkedList<String>();

            links.add(uriField.getText());
            System.out.println("added to list ....");

            while ((links.size() > 0) && (contor < maxSize)) {
                String aux = links.poll();
                if (getOverride) {
                    parseIt(aux);
                } else {
                    parseItGetOpt(aux);
                }
                currentFile = GetFileName(aux);
                // currentFile = aux.replaceAll('\\',' ');
                //  currentFile = currentFile.replaceAll(aux, redirect)
                System.out.println(" contor =" + contor);
                System.out.println("poped ...." + aux);
                System.out.println("link list size  =" + links.size());
                contor++;
//
            }
            BuildReverseIndex builder = new BuildReverseIndex(list);
            builder.MakeIndexFiles();
        }
        contor = 0;


    }//GEN-LAST:event_GoButtonActionPerformed

    private void DisplayTitleActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_DisplayTitleActionPerformed
        // TODO add your handling code here:
    }//GEN-LAST:event_DisplayTitleActionPerformed

    private void uriFieldActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_uriFieldActionPerformed
        // TODO add your handling code here:
    }//GEN-LAST:event_uriFieldActionPerformed

    private void MultiSearch() {

        String searchString = SearchBox.getText().trim();
        Display.append(" \n Rezultate Cautare : \n");
        String[] searchArray = searchString.split(" ");
        ArrayList<String> operatorArray = new ArrayList<String>();
        ArrayList<String> termArray = new ArrayList<String>();
        int operatorCount = 1;
        operatorArray.add("or");
        termArray.add(searchArray[0]);
        for (int i = 1; i < searchArray.length; i++) {
            if (searchArray[i].equalsIgnoreCase("or")) {

                operatorArray.add(searchArray[i].toLowerCase().trim());
                i++;
                termArray.add(searchArray[i].toLowerCase().trim());
                operatorCount++;

            } else {
                if (searchArray[i].equalsIgnoreCase("not")) {

                    operatorArray.add(searchArray[i].toLowerCase().trim());

                    termArray.add(searchArray[i + 1].toLowerCase().trim());
                    i++;
                    operatorCount++;

                } else {
                    //implicit and

                    operatorArray.add(searchArray[i].toLowerCase().trim());
                    termArray.add(searchArray[i].toLowerCase().trim());
                    operatorCount++;
                }
            }
        }
        ArrayList<Integer> docList = new ArrayList<Integer>();
        for (int i = 0; i < operatorCount; i++) {
            int[] wordResult;
            if (operatorArray.get(i).contains("or")) {
                wordResult = Search(termArray.get(i));
                for (int j = 0; j < wordResult.length; j++) {
                    if (!docList.contains(wordResult[j])) {
                        docList.add(wordResult[j]);
                    }
                }
            } else {
                if (operatorArray.get(i).contains("not")) {
                    wordResult = Search(termArray.get(i));
                    for (int j = 0; j < wordResult.length; j++) {
                        if (docList.contains(wordResult[j])) {
                            docList.remove(docList.indexOf(wordResult[j]));

                        }
                    }
                } else {
                    wordResult = Search(termArray.get(i));
                    ArrayList<Integer> docAux = new ArrayList<Integer>();
                    for (int j = 0; j < wordResult.length; j++) {
                        if (docList.contains(wordResult[j])) {
                            docAux.add(wordResult[j]);
                        }
                    }
                    docList = docAux;
                }
            }
        }
        int[] arr = new int[docList.size()];
        for (int i = 0; i < docList.size(); i++) {
            Display.append("found " + docList.get(i) + "|| " + (i + 1) + "  of " + docList.size() + "results total \n");
            arr[i] = docList.get(i);
        }
        ArrayList<String> list = BuildReverseIndex.GetFileNames(arr);
        for (int i = 0; i < list.size(); i++) {
            Display.append("found " + list.get(i) + "|| " + (i + 1) + "  of " + list.size() + "results total  \n");
        }

    }

    private int[] Search(String word) {
        try {
            System.out.println("inside search ...1");
            SearchInInverseIndex search = new SearchInInverseIndex(" inverserIndex.txt", " inverserIndex.txt coding");
            System.out.println("inside search ...2");
            search.searchWord(word);
            System.out.println("inside search ...3");
            //    for (int i = 0; i < search.nr_words; i++) {
            //        System.out.println("inside search ..." + (i + 3));
            //        System.out.println(" wordId    " + search.searchResponseDocumentList[i]);
            //   }
            //    System.out.println("wanted hashcode " + SearchBox.getText().hashCode());
            // for (int i = 0; i < search.nr_words; i++) {
            //         System.out.println("->" + search.searchResponseDocumentList[i]);
            ///     }
            int[] p = search.getDocumentIDs(search.searchResponseDocumentList);
            //      for (int i = 0; i < p.length; i++) {
            ///      System.out.println(" similar to " + p[i]);
            //    }
            return p;
        //search.printSimilars(SearchBox.getText().trim());
        } catch (Exception ex) {
            System.out.println("excetion ++");
            System.out.println(ex.getMessage());
            ex.printStackTrace();
            return new int[0];
        }
    }
    private void SearchBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_SearchBtnActionPerformed
        // TODO add your handling code here:
        try {
            Display.setText("  \n ");
            MultiSearch();
//            System.out.println("inside search ...1");
//            SearchInInverseIndex search = new SearchInInverseIndex(" inverserIndex.txt", " inverserIndex.txt coding");
//            System.out.println("inside search ...2");
//            search.searchWord(SearchBox.getText());
//            System.out.println("inside search ...3");
//            for (int i = 0; i < search.nr_words; i++) {
//                System.out.println("inside search ..." + (i + 3));
//                System.out.println(" wordId    " + search.searchResponseDocumentList[i]);
//            }
//            System.out.println("wanted hashcode " + SearchBox.getText().hashCode());
//            for (int i = 0; i < search.nr_words; i++) {
//                System.out.println("->" + search.searchResponseDocumentList[i]);
//            }
//            int[] p = search.getDocumentIDs(search.searchResponseDocumentList);
//            for (int i = 0; i < p.length; i++) {
//                System.out.println(" similar to " + p[i]);
//            }

        //search.printSimilars(SearchBox.getText().trim());
        } catch (Exception ex) {
            System.out.println("excetion ++");
            System.out.println(ex.getMessage());
            ex.printStackTrace();
        }
}//GEN-LAST:event_SearchBtnActionPerformed
    /**
     * @param args the command line arguments
     */
//    public static void main(String args[]) {
//        java.awt.EventQueue.invokeLater(new Runnable() {
//
//            public void run() {
//                new UI().setVisible(true);
//            }
//        });
//    }

    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JTextArea Display;
    private javax.swing.JTextArea DisplayLinks;
    private javax.swing.JTextArea DisplayMeta;
    private javax.swing.JTextArea DisplayText;
    private javax.swing.JTextField DisplayTitle;
    private javax.swing.JTextField DnsAddressBox;
    private javax.swing.JButton GoButton;
    private javax.swing.JTextField SearchBox;
    private javax.swing.JButton SearchBtn;
    private javax.swing.JCheckBox getOverrideBox;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JLabel jLabel2;
    private javax.swing.JScrollPane jScrollPane1;
    private javax.swing.JScrollPane jScrollPane2;
    private javax.swing.JScrollPane jScrollPane3;
    private javax.swing.JScrollPane jScrollPane4;
    private javax.swing.JTextField maxDepthField1;
    private javax.swing.JCheckBox proxyBox;
    private javax.swing.JTextField proxyField;
    private javax.swing.JTextField uriField;
    // End of variables declaration//GEN-END:variables
}
